***************
*** Table 2 ***
***************
global db "D:\Dropbox\unequal_gains\main_data"
global resultspath "D:\Dropbox\unequal_gains\QJE revision plan\analysis\clean_results"

****************************
** Part 1: RMS Inflation ***
****************************

* (1) across barcodes
clear
set obs 1 
generate tornqvist_all_price_index=.
save "$db/Important Price Datasets/inflation_income_acrossUPC_RMS.dta", replace

foreach i of numlist 2006(1)2014 {

display "Starting Tornqvist inflation difference for year `i', QM"

local var=`i'+1

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear
rename total_quantity total_quantity_old
rename average_unit_price average_unit_price_old
rename average_unit_price_all average_unit_price_all_old

merge 1:1 upc upc_ver income_quintile using "$db/Important Price Datasets/price_index_household_income_`var'_2004_modules_final.dta"
keep if _merge==3
drop total_spending
drop _merge

* now get price ratio from RMS data
merge m:1 upc upc_ver_uc using "$db/Important Price Datasets/inflation_upc_`i'_RMS.dta"

* get rid of outliers
sum price_ratio, d
drop if price_ratio<r(p1) | price_ratio>r(p99)
sum price_ratio_all, d
drop if price_ratio_all<r(p1) | price_ratio_all>r(p99)

* compute UPC spending/shares by quintiles: 
bysort income_quintile: egen double total_spending=sum(average_unit_price*total_quantity)
bysort income_quintile: egen double total_spending_old=sum(average_unit_price_old*total_quantity_old)
gen double share=average_unit_price*total_quantity/total_spending
gen double share_old=average_unit_price_old*total_quantity_old/total_spending_old

* Across UPCs 
gen weight=1/2*(share+share_old)
gen double numerator_temp=weight*log(price_ratio_all)
bysort income_quintile: egen double numerator=sum(numerator_temp)
gen double tornqvist_all_price_index=exp(numerator)
drop weight numerator numerator_temp

keep tornqvist_all_price_index income_quintile
duplicates drop

gen year=`var'

append using "$db/Important Price Datasets/inflation_income_acrossUPC_RMS.dta"
drop if missing(tornqvist_all_price_index)
save "$db/Important Price Datasets/inflation_income_acrossUPC_RMS.dta", replace 

}

use "$db/Important Price Datasets/inflation_income_acrossUPC_RMS.dta", clear
foreach i in tornqvist {
gen `i'_infl=(`i'_all_price-1)*100
}
collapse (mean) *_infl, by(income_quintile)
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen infl_diff = tornqvist_infl-tornqvist_infl[_n-1]
drop if missing(infl_diff)
keep infl_diff
gen AggregationLevel="Barcodes"
save "$resultspath\Table2_RMS.dta", replace


* (2) across modules by quality deciles (QM)
clear
set obs 1 
generate tornqvist_price_index_inc=.
save "$db/Important Price Datasets/inflation_income_acrossQM_RMS.dta", replace

foreach i of numlist 2006(1)2014 {

display "Starting Tornqvist inflation difference for year `i', QM"

* first, compute spending shares in each module by quality deciles 
local var=`i'+1

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear
rename total_quantity total_quantity_old
rename average_unit_price average_unit_price_old
rename average_unit_price_all average_unit_price_all_old

merge 1:1 upc upc_ver income_quintile using "$db/Important Price Datasets/price_index_household_income_`var'_2004_modules_final.dta"
keep if _merge==3
drop total_spending
drop _merge

* bring in info on UPC price decile (within product module) 
merge m:1 upc upc_ver_uc using "$db/Important Lists/upc_brand_size_final.dta"
keep if _merge==3
drop _merge
sum size1_amount, d
drop if size1_amount<r(p1) | size1_amount>r(p99)
gen average_unit_price_adj=(average_unit_price_all+average_unit_price_all_old)/(2*size1_amount*multi)
* create quality ranks without weights
sort product_module_code average_unit_price_adj
bysort product_module_code: gen double temp=[_N]
bysort product_module_code: gen double temp2=[_n]
gen rank=temp2/temp*100
gen quality_rank=.
foreach r of numlist 1(1)10 {
replace quality_rank=`r' if rank<=`r'*10 & missing(quality_rank)
}

gen double price_ratio=average_unit_price/average_unit_price_old
gen double price_ratio_all=average_unit_price_all/average_unit_price_all_old
* get rid of outliers
sum price_ratio, d
drop if price_ratio<r(p1) | price_ratio>r(p99)
*sum price_ratio_all, d
*drop if price_ratio_all<r(p1) | price_ratio_all>r(p99)

* compute spending shares by product module - quality
bysort product_module quality income_quintile: egen double total_spending_inc_QM=sum(average_unit_price*total_quantity)
bysort product_module quality income_quintile: egen double total_spending_inc_QM_old=sum(average_unit_price_old*total_quantity_old)

keep product_module quality income_quintile total_spending_inc_QM total_spending_inc_QM_old
duplicates drop

* second, bring to file with inflation by quality-modules
merge m:1 product_module quality using "$db/Important Price Datasets/inflation_QM_`i'_2yrs_RMS.dta"
keep if _merge==3

* now compute Tornqvist inflation
bysort income_quintile: egen double total_spending_inc=sum(total_spending_inc_QM)
bysort income_quintile: egen double total_spending_inc_old=sum(total_spending_inc_QM_old)
gen double share_inc_QM=total_spending_inc_QM/total_spending_inc
gen double share_inc_QM_old=total_spending_inc_QM_old/total_spending_inc_old
gen weight_inc_QM=1/2*(share_inc_QM+share_inc_QM_old)
gen double numerator_temp=weight*log(tornqvist)
bysort income_quintile: egen double numerator=sum(numerator_temp)
gen double tornqvist_price_index_inc=exp(numerator)
drop numerator numerator_temp weight

keep tornqvist_price_index_inc income_quintile
duplicates drop
gen year=`var'

append using "$db/Important Price Datasets/inflation_income_acrossQM_RMS.dta"
drop if missing(tornqvist_price_index_inc)
save "$db/Important Price Datasets/inflation_income_acrossQM_RMS.dta", replace 
}

use "$db/Important Price Datasets/inflation_income_acrossQM_RMS.dta", clear
foreach i in tornqvist {
gen `i'_infl=(`i'_price-1)*100
}
collapse (mean) *_infl, by(income_quintile)
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen infl_diff = tornqvist_infl-tornqvist_infl[_n-1]
drop if missing(infl_diff)
keep infl_diff
gen AggregationLevel="Product Modules by Quality Deciles"
append using "$resultspath\Table2_RMS.dta"
save "$resultspath\Table2_RMS.dta", replace

** (3) Across Modules (M)
clear
set obs 1 
generate tornqvist_price_index_inc=.
save "$db/Important Price Datasets/inflation_income_acrossM_RMS.dta", replace

foreach i of numlist 2006(1)2014 {

display "Starting Tornqvist inflation difference for year `i', M"

local var=`i'+1

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear
rename total_quantity total_quantity_old
rename average_unit_price average_unit_price_old
rename average_unit_price_all average_unit_price_all_old

merge 1:1 upc upc_ver income_quintile using "$db/Important Price Datasets/price_index_household_income_`var'_2004_modules_final.dta"
keep if _merge==3
drop total_spending

gen double price_ratio=average_unit_price/average_unit_price_old
gen double price_ratio_all=average_unit_price_all/average_unit_price_all_old
* get rid of outliers
sum price_ratio, d
drop if price_ratio<r(p1) | price_ratio>r(p99)
sum price_ratio_all, d
drop if price_ratio_all<r(p1) | price_ratio_all>r(p99)

* compute spending shares by product module - quality
bysort product_module income_quintile: egen double total_spending_inc_M=sum(average_unit_price*total_quantity)
bysort product_module income_quintile: egen double total_spending_inc_M_old=sum(average_unit_price_old*total_quantity_old)

keep product_module income_quintile total_spending_inc_M total_spending_inc_M_old
duplicates drop
merge m:1 product_module using "$db/Important Price Datasets/inflation_M_`i'_RMS.dta"
keep if _merge==3

* now compute Tornqvist inflation
bysort income_quintile: egen double total_spending_inc=sum(total_spending_inc_M)
bysort income_quintile: egen double total_spending_inc_old=sum(total_spending_inc_M_old)
gen double share_inc_M=total_spending_inc_M/total_spending_inc
gen double share_inc_M_old=total_spending_inc_M_old/total_spending_inc_old
gen weight_inc_M=1/2*(share_inc_M+share_inc_M_old)
gen double numerator_temp=weight*log(tornqvist)
bysort income_quintile: egen double numerator=sum(numerator_temp)
gen double tornqvist_price_index_inc=exp(numerator)
drop numerator numerator_temp weight

keep tornqvist_price_index_inc income_quintile
duplicates drop
gen year=`var'

append using "$db/Important Price Datasets/inflation_income_acrossM_RMS.dta"
drop if missing(tornqvist_price_index_inc)
save "$db/Important Price Datasets/inflation_income_acrossM_RMS.dta", replace 
}

use "$db/Important Price Datasets/inflation_income_acrossM_RMS.dta", clear
foreach i in tornqvist {
gen `i'_infl=(`i'_price-1)*100
}
collapse (mean) *_infl, by(income_quintile)
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen infl_diff = tornqvist_infl-tornqvist_infl[_n-1]
drop if missing(infl_diff)
keep infl_diff
gen AggregationLevel="Product Modules"
append using "$resultspath\Table2_RMS.dta"
save "$resultspath\Table2_RMS.dta", replace


** (4) Across groups (G)
clear
set obs 1 
generate tornqvist_price_index_inc=.
save "$db/Important Price Datasets/inflation_income_acrossG_RMS.dta", replace

foreach i of numlist 2006(1)2014 {

display "Starting Tornqvist inflation difference for year `i', G"

local var=`i'+1

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear
rename total_quantity total_quantity_old
rename average_unit_price average_unit_price_old
rename average_unit_price_all average_unit_price_all_old

merge 1:1 upc upc_ver income_quintile using "$db/Important Price Datasets/price_index_household_income_`var'_2004_modules_final.dta"
keep if _merge==3
drop total_spending

gen double price_ratio=average_unit_price/average_unit_price_old
gen double price_ratio_all=average_unit_price_all/average_unit_price_all_old
* get rid of outliers
sum price_ratio, d
drop if price_ratio<r(p1) | price_ratio>r(p99)
sum price_ratio_all, d
drop if price_ratio_all<r(p1) | price_ratio_all>r(p99)

* compute spending shares by product module - quality
bysort product_group income_quintile: egen double total_spending_inc_G=sum(average_unit_price*total_quantity)
bysort product_group income_quintile: egen double total_spending_inc_G_old=sum(average_unit_price_old*total_quantity_old)

keep product_group income_quintile total_spending_inc_G total_spending_inc_G_old
duplicates drop
merge m:1 product_group using "$db/Important Price Datasets/inflation_G_`i'_RMS.dta"
keep if _merge==3

* now compute Tornqvist inflation
bysort income_quintile: egen double total_spending_inc=sum(total_spending_inc_G)
bysort income_quintile: egen double total_spending_inc_old=sum(total_spending_inc_G_old)
gen double share_inc_G=total_spending_inc_G/total_spending_inc
gen double share_inc_G_old=total_spending_inc_G_old/total_spending_inc_old
gen weight_inc_G=1/2*(share_inc_G+share_inc_G_old)
gen double numerator_temp=weight*log(tornqvist)
bysort income_quintile: egen double numerator=sum(numerator_temp)
gen double tornqvist_price_index_inc=exp(numerator)
drop numerator numerator_temp weight

keep tornqvist_price_index_inc income_quintile
duplicates drop
gen year=`var'

append using "$db/Important Price Datasets/inflation_income_acrossG_RMS.dta"
drop if missing(tornqvist_price_index_inc)
save "$db/Important Price Datasets/inflation_income_acrossG_RMS.dta", replace 
}

use "$db/Important Price Datasets/inflation_income_acrossG_RMS.dta", clear
foreach i in tornqvist {
gen `i'_infl=(`i'_price-1)*100
}
collapse (mean) *_infl, by(income_quintile)
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen infl_diff = tornqvist_infl-tornqvist_infl[_n-1]
drop if missing(infl_diff)
keep infl_diff
gen AggregationLevel="Product Groups"
append using "$resultspath\Table2_RMS.dta"
save "$resultspath\Table2_RMS.dta", replace

** (5) Across departments (D)
clear
set obs 1 
generate tornqvist_price_index_inc=.
save "$db/Important Price Datasets/inflation_income_acrossD_RMS.dta", replace

foreach i of numlist 2006(1)2014 {

display "Starting Tornqvist inflation difference for year `i', D"

local var=`i'+1

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear
rename total_quantity total_quantity_old
rename average_unit_price average_unit_price_old
rename average_unit_price_all average_unit_price_all_old

merge 1:1 upc upc_ver income_quintile using "$db/Important Price Datasets/price_index_household_income_`var'_2004_modules_final.dta"
keep if _merge==3
drop total_spending

gen double price_ratio=average_unit_price/average_unit_price_old
gen double price_ratio_all=average_unit_price_all/average_unit_price_all_old
* get rid of outliers
sum price_ratio, d
drop if price_ratio<r(p1) | price_ratio>r(p99)
sum price_ratio_all, d
drop if price_ratio_all<r(p1) | price_ratio_all>r(p99)

* compute spending shares by product module - quality
bysort department income_quintile: egen double total_spending_inc_D=sum(average_unit_price*total_quantity)
bysort department income_quintile: egen double total_spending_inc_D_old=sum(average_unit_price_old*total_quantity_old)

keep department income_quintile total_spending_inc_D total_spending_inc_D_old
duplicates drop
merge m:1 department using "$db/Important Price Datasets/inflation_D_`i'_RMS.dta"
keep if _merge==3

* now compute Tornqvist inflation
bysort income_quintile: egen double total_spending_inc=sum(total_spending_inc_D)
bysort income_quintile: egen double total_spending_inc_old=sum(total_spending_inc_D_old)
gen double share_inc_D=total_spending_inc_D/total_spending_inc
gen double share_inc_D_old=total_spending_inc_D_old/total_spending_inc_old
gen weight_inc_D=1/2*(share_inc_D+share_inc_D_old)
gen double numerator_temp=weight*log(tornqvist)
bysort income_quintile: egen double numerator=sum(numerator_temp)
gen double tornqvist_price_index_inc=exp(numerator)
drop numerator numerator_temp weight

keep tornqvist_price_index_inc income_quintile
duplicates drop
gen year=`var'

append using "$db/Important Price Datasets/inflation_income_acrossD_RMS.dta"
drop if missing(tornqvist_price_index_inc)
save "$db/Important Price Datasets/inflation_income_acrossD_RMS.dta", replace 
}

use "$db/Important Price Datasets/inflation_income_acrossD_RMS.dta", clear
foreach i in tornqvist {
gen `i'_infl=(`i'_price-1)*100
}
collapse (mean) *_infl, by(income_quintile)
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen infl_diff = tornqvist_infl-tornqvist_infl[_n-1]
drop if missing(infl_diff)
keep infl_diff
gen AggregationLevel="Departments"
append using "$resultspath\Table2_RMS.dta"
save "$resultspath\Table2_RMS.dta", replace

* finalize table
use "$resultspath\Table2_RMS.dta", clear
egen temp=max(infl_diff)
gen share=infl_diff/temp*100
drop temp
format infl %8.4fc
format share %8.1fc
gsort - infl_diff
order Agg infl share
save "$resultspath\Table2_RMS.dta", replace


****************************
** Part 2: RMS New Goods ***
****************************


** (1) Across barcodes: 
* This is by definition the same as in HMS, except that we have to start post 2006
clear
set obs 1 
generate year=.
save "$db/Important Price Datasets/RMSnew_exit_income_acrossUPC.dta", replace

foreach i of numlist 2006(1)2015 {

local prev=`i'-1

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear

merge m:1 upc upc_ver using "$db/Important Lists/available_upcs_upto`prev'_final.dta"
drop if _merge==2
gen new=(_merge==1)
drop _merge

merge m:1 upc upc_ver using "$db/Important Lists/available_upcs_post`i'_final.dta"
drop if _merge==2
gen exit=(_merge==1)

drop total_spending*

* compute overall spending shares on new/old by quintiles: 
bysort income_quintile: egen double total_spending=sum(average_unit_price*total_quantity)
bysort income_quintile: egen double total_spending_new=sum(average_unit_price*total_quantity*new)
bysort income_quintile: egen double total_spending_exit=sum(average_unit_price*total_quantity*exit)

gen ssnp  = total_spending_new/total_spending
gen ssep  = total_spending_exit/total_spending

keep income_q ssnp* ssep* 
duplicates drop 

gen year=`i'
append using "$db/Important Price Datasets/RMSnew_exit_income_acrossUPC.dta"
drop if missing(year)
save "$db/Important Price Datasets/RMSnew_exit_income_acrossUPC.dta", replace
}

use "$db/Important Price Datasets/RMSnew_exit_income_acrossUPC.dta", clear
keep ssnp ssep income_quintile year
rename ssnp ssnp 
rename ssep ssep_temp

tsset income_q year
gen ssep=L.ssep_temp
drop ssep_temp

drop if year==2006
order ssnp ssep
gen feenstra=(1-ssnp)/(1-ssep)

collapse (mean) ssnp ssep feenstra, by(income_quintile)
replace ssnp=ssnp*100
replace ssep=ssep*100
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen log_feenstra=log(feenstra)
foreach i in ssnp ssep log_feenstra {
gen `i'_diff=`i'-`i'[_n-1]
}
drop if missing(ssnp_diff) 
keep *_diff
replace log_feenstra=log_feenstra*100
gen AggregationLevel="Barcodes"
merge 1:1 Aggregation using "$resultspath\Table2_RMS.dta"
drop _merge
save "$resultspath\Table2_RMS.dta", replace

** (2) Across QM:
clear
set obs 1 
generate year=.
save "$db/Important Price Datasets/RMSnew_exit_income_acrossQM.dta", replace

* note that here we need to have data on following year to be able to build the price ratio
foreach i of numlist 2006(1)2015 {

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear
rename average_unit_price_all average_unit_price_all_old

* bring in info on UPC price decile (within product module):
merge m:1 upc upc_ver_uc using "$db/Important Lists/upc_brand_size_final.dta"
keep if _merge==3
drop _merge
sum size1_amount, d
drop if size1_amount<r(p1) | size1_amount>r(p99)
gen average_unit_price_adj=average_unit_price_all/(size1_amount*multi)
* create quality ranks without weights 
sort product_module_code average_unit_price_adj
bysort product_module_code: gen double temp=[_N]
bysort product_module_code: gen double temp2=[_n]
gen rank=temp2/temp*100
gen quality_rank=.
foreach r of numlist 1(1)10 {
replace quality_rank=`r' if rank<=`r'*10 & missing(quality_rank)
}

* compute overall spending shares on modules by quality rank
drop total_spending*
bysort product_module quality income_quintile: egen double total_spendingQM=sum(average_unit_price*total_quantity)
bysort income_quintile: egen double total_spending=sum(average_unit_price*total_quantity)
gen shareQM=total_spendingQM/total_spending

* compute spending shares 
keep product_module quality shareQM income_quintile
duplicates drop
merge m:1 product_module quality using "$db/Important Price Datasets/RMSnew_exit_QM_`i'"
keep if _merge==3

bysort income_quintile: egen ssnp_avg_QM=sum(ssnpQM*shareQM)
bysort income_quintile: egen ssep_avg_QM=sum(ssepQM*shareQM)
keep income_quintile *_avg_QM
duplicates drop
gen year=`i'

append using "$db/Important Price Datasets/RMSnew_exit_income_acrossQM.dta"
drop if missing(year)
save "$db/Important Price Datasets/RMSnew_exit_income_acrossQM.dta", replace
}

use "$db/Important Price Datasets/RMSnew_exit_income_acrossQM.dta", clear
keep ssnp ssep income_quintile year
rename ssnp ssnp 
rename ssep ssep_temp

tsset income_q year
gen ssep=L.ssep_temp
drop ssep_temp

drop if year==2006
order ssnp ssep
gen feenstra=(1-ssnp)/(1-ssep)

collapse (mean) ssnp ssep feenstra, by(income_quintile)
replace ssnp=ssnp*100
replace ssep=ssep*100
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen log_feenstra=log(feenstra)
foreach i in ssnp ssep log_feenstra {
gen `i'_diff=`i'-`i'[_n-1]
}
drop if missing(ssnp_diff) 
keep *_diff
replace log_feenstra=log_feenstra*100
gen AggregationLevel="Product Modules by Quality Deciles"
merge 1:1 Aggregation using "$resultspath\Table2_RMS.dta"
drop _merge
save "$resultspath\Table2_RMS.dta", replace

** (3) Across M:
clear
set obs 1 
generate year=.
save "$db/Important Price Datasets/RMSnew_exit_income_acrossM.dta", replace

foreach i of numlist 2006(1)2015 {

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear

* compute overall spending shares on modules by quality rank
drop total_spending*
bysort product_module income_quintile: egen double total_spendingM=sum(average_unit_price*total_quantity)
bysort income_quintile: egen double total_spending=sum(average_unit_price*total_quantity)
gen shareM=total_spendingM/total_spending

* compute spending shares 
keep product_module shareM income_quintile
duplicates drop
merge m:1 product_module using "$db/Important Price Datasets/RMSnew_exit_M_`i'"
keep if _merge==3

bysort income_quintile: egen ssnp_avg_M=sum(ssnpM*shareM)
bysort income_quintile: egen ssep_avg_M=sum(ssepM*shareM)
keep income_quintile *_avg_M
duplicates drop
gen year=`i'

append using "$db/Important Price Datasets/RMSnew_exit_income_acrossM.dta"
drop if missing(year)
save "$db/Important Price Datasets/RMSnew_exit_income_acrossM.dta", replace
}

use "$db/Important Price Datasets/RMSnew_exit_income_acrossM.dta", clear
keep ssnp ssep income_quintile year
rename ssnp ssnp 
rename ssep ssep_temp

tsset income_q year
gen ssep=L.ssep_temp
drop ssep_temp

drop if year==2006
order ssnp ssep
gen feenstra=(1-ssnp)/(1-ssep)

collapse (mean) ssnp ssep feenstra, by(income_quintile)
replace ssnp=ssnp*100
replace ssep=ssep*100
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen log_feenstra=log(feenstra)
foreach i in ssnp ssep log_feenstra {
gen `i'_diff=`i'-`i'[_n-1]
}
drop if missing(ssnp_diff) 
keep *_diff
replace log_feenstra=log_feenstra*100
gen AggregationLevel="Product Modules"
merge 1:1 Aggregation using "$resultspath\Table2_RMS.dta"
drop _merge
save "$resultspath\Table2_RMS.dta", replace


** (4) Across G:
clear
set obs 1 
generate year=.
save "$db/Important Price Datasets/RMSnew_exit_income_acrossG.dta", replace

foreach i of numlist 2006(1)2015 {

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear

* compute overall spending shares on product groups
drop total_spending*
bysort product_group income_quintile: egen double total_spendingG=sum(average_unit_price*total_quantity)
bysort income_quintile: egen double total_spending=sum(average_unit_price*total_quantity)
gen shareG=total_spendingG/total_spending

* compute spending shares 
keep product_group shareG income_quintile
duplicates drop
merge m:1 product_group using "$db/Important Price Datasets/RMSnew_exit_G_`i'"
keep if _merge==3

bysort income_quintile: egen ssnp_avg_G=sum(ssnpG*shareG)
bysort income_quintile: egen ssep_avg_G=sum(ssepG*shareG)
keep income_quintile *_avg_G
duplicates drop
gen year=`i'

append using "$db/Important Price Datasets/RMSnew_exit_income_acrossG.dta"
drop if missing(year)
save "$db/Important Price Datasets/RMSnew_exit_income_acrossG.dta", replace
}

use "$db/Important Price Datasets/RMSnew_exit_income_acrossG.dta", clear
keep ssnp ssep income_quintile year
rename ssnp ssnp 
rename ssep ssep_temp

tsset income_q year
gen ssep=L.ssep_temp
drop ssep_temp

drop if year==2006
order ssnp ssep
gen feenstra=(1-ssnp)/(1-ssep)

collapse (mean) ssnp ssep feenstra, by(income_quintile)
replace ssnp=ssnp*100
replace ssep=ssep*100
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen log_feenstra=log(feenstra)
foreach i in ssnp ssep log_feenstra {
gen `i'_diff=`i'-`i'[_n-1]
}
drop if missing(ssnp_diff) 
keep *_diff
replace log_feenstra=log_feenstra*100
gen AggregationLevel="Product Groups"
merge 1:1 Aggregation using "$resultspath\Table2_RMS.dta"
drop _merge
save "$resultspath\Table2_RMS.dta", replace

** (5) Across D:
clear
set obs 1 
generate year=.
save "$db/Important Price Datasets/RMSnew_exit_income_acrossD.dta", replace

foreach i of numlist 2006(1)2015 {

use "$db/Important Price Datasets/price_index_household_income_`i'_2004_modules_final.dta", clear

* compute overall spending shares on departments
drop total_spending*
bysort department income_quintile: egen double total_spendingD=sum(average_unit_price*total_quantity)
bysort income_quintile: egen double total_spending=sum(average_unit_price*total_quantity)
gen shareD=total_spendingD/total_spending

* compute spending shares 
keep department shareD income_quintile
duplicates drop
merge m:1 department_code using "$db/Important Price Datasets/RMSnew_exit_D_`i'"
keep if _merge==3

bysort income_quintile: egen ssnp_avg_D=sum(ssnpD*shareD)
bysort income_quintile: egen ssep_avg_D=sum(ssepD*shareD)
keep income_quintile *_avg_D
duplicates drop
gen year=`i'

append using "$db/Important Price Datasets/RMSnew_exit_income_acrossD.dta"
drop if missing(year)
save "$db/Important Price Datasets/RMSnew_exit_income_acrossD.dta", replace
}

use "$db/Important Price Datasets/RMSnew_exit_income_acrossD.dta", clear
keep ssnp ssep income_quintile year
rename ssnp ssnp 
rename ssep ssep_temp

tsset income_q year
gen ssep=L.ssep_temp
drop ssep_temp

drop if year==2006
order ssnp ssep
gen feenstra=(1-ssnp)/(1-ssep)

collapse (mean) ssnp ssep feenstra, by(income_quintile)
replace ssnp=ssnp*100
replace ssep=ssep*100
keep if income_quintile==1 | income_quintile==5
gsort - income_quintile
gen log_feenstra=log(feenstra)
foreach i in ssnp ssep log_feenstra {
gen `i'_diff=`i'-`i'[_n-1]
}
drop if missing(ssnp_diff) 
keep *_diff
replace log_feenstra=log_feenstra*100
gen AggregationLevel="Departments"
merge 1:1 Aggregation using "$resultspath\Table2_RMS.dta"
drop _merge
save "$resultspath\Table2_RMS.dta", replace


* finalize table 
use "$resultspath\Table2_RMS.dta", clear
egen temp=max(log_feenstra)
gen share_feenstra=log_feenstra/temp*100
drop temp
rename share share_infl
order AggregationLevel infl_diff share_infl log_feenstra_diff share_f
gsort - log_feenstra
format *_diff %8.3fc
format share* %8.1fc
save "$resultspath\Table2_RMS.dta", replace
